# -*- coding: utf8 -*-

import scrapy
import json
#from lagouspider.items import BigDataJobsItem
from scrapy.selector import Selector


class BigdatajobsSpider(scrapy.Spider):
    name = "BigDataJobs"
    # allowed_domains = ["https://www.lagou.com/jobs/list_%E5%A4%A7%E6%95%B0%E6%8D%AE?labelWords=&fromSearch=true&suginput="]



    """
    start url 是这个
    https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=
    一步步解析

    """



    start_urls = ['https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=']

    #url = 'https://www.lagou.com/jobs/positionAjax.json?city=%E5%B9%BF%E5%B7%9E&needAddtionalResult=false'
    # url = 'http://www.lagou.com/jobs/positionAjax.json?'

    # totalPage = 0
    # page = 1
    # kd = u'大数据'
    # def start_requests(self):
    #     return [scrapy.http.FormRequest(self.url,
    #                                     formdata={'pn':str(self.page),'kd':self.kd},callback=self.parse)]

    def parse(self, response):
        #item = BigDataJobsItem
        sel = Selector(response)
        job_url = sel.xpath(".//*[@id='s_position_list']/ul/li/div[1]/div[1]/div[1]/a/@href").extract()
        print (job_url)

        # alllist = json.loads(response.body)
        # content = alllist["content"]
        # positionResult = content["positionResult"]
        # joblist = positionResult["result"]
        # self.totalPage = positionResult['totalCount']/15 +1;
        # for job in joblist :
        #     # print type(job['companyFullName'])
        #     # a = job['companyFullName'.decode('utf8')]
        #     # print a
        #     item['com_name'] = job['companyFullName'.decode('utf8')]
        #     # item['job_name'] = job['positionName']
        #     # item['salary'] = job['salary']
        #     # item['job_attract'] = job['positionAdvantage']
        #     # item['exp_require'] = job['workYear']
        #     # item['workplace'] =[]
        #     # item['edu_require'] =[]
        #     # item['job_describe'] =[]
        #     yield item



#1目前问题，job里的数据无法导入item里，老师说要导入具体的值而不是一个对象？
#2问题是网上找的模板可以导入，又好像没有区别
#3翻页问题：totalCount/15+1,这里的totalCount提取出的好像不是具体的广州大数据里的totalCount
#4其他具体内容体思路:提取positionId,已测试提取成功，具体内容的url格式为："https://www.lagou.com/jobs/positionId.html"


#或许问题1和3是有关联的，因此返回的companyFullName并不是一个具体的值？，可以试下print查看和实际有无区别


# 解决：
# 3：的确json文件地址中只有city，所提取内容没有引用关键字‘大数据’，引用后totalCount/15 + 1=8


